{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(569, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension    ...      worst texture  worst perimeter  \\\n",
       "0                 0.07871    ...              17.33           184.60   \n",
       "1                 0.05667    ...              23.41           158.80   \n",
       "2                 0.05999    ...              25.53           152.50   \n",
       "3                 0.09744    ...              26.50            98.87   \n",
       "4                 0.05883    ...              16.67           152.20   \n",
       "\n",
       "   worst area  worst smoothness  worst compactness  worst concavity  \\\n",
       "0      2019.0            0.1622             0.6656           0.7119   \n",
       "1      1956.0            0.1238             0.1866           0.2416   \n",
       "2      1709.0            0.1444             0.4245           0.4504   \n",
       "3       567.7            0.2098             0.8663           0.6869   \n",
       "4      1575.0            0.1374             0.2050           0.4000   \n",
       "\n",
       "   worst concave points  worst symmetry  worst fractal dimension  diagnosis  \n",
       "0                0.2654          0.4601                  0.11890          0  \n",
       "1                0.1860          0.2750                  0.08902          0  \n",
       "2                0.2430          0.3613                  0.08758          0  \n",
       "3                0.2575          0.6638                  0.17300          0  \n",
       "4                0.1625          0.2364                  0.07678          0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "\n",
    "data = load_breast_cancer()\n",
    "train_df = pd.DataFrame(data.data, columns=data.feature_names)\n",
    "train_df[\"diagnosis\"] = data.target\n",
    "\n",
    "print(train_df.shape)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Experiment Key:   'LOKr-L2lRGe8e8l0E-8SY5FEcl2u7dwBRtrl36HDjvQ='\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import Environment, CVExperiment\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "env = Environment(\n",
    "    train_dataset=train_df,\n",
    "    results_path=\"HyperparameterHunterAssets\",\n",
    "    target_column=\"diagnosis\",\n",
    "    metrics=[\"roc_auc_score\"],\n",
    "    cv_type=StratifiedKFold,\n",
    "    cv_params=dict(n_splits=10, shuffle=True, random_state=32),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that HyperparameterHunter has an active `Environment`, we can do two things:\n",
    "\n",
    "# 1. Perform Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<18:54:43> Validated Environment:  'LOKr-L2lRGe8e8l0E-8SY5FEcl2u7dwBRtrl36HDjvQ='\n",
      "<18:54:43> Initialized Experiment: 'c3ddb52c-ac47-46bd-9b11-f6407f7ca611'\n",
      "<18:54:43> Hyperparameter Key:     'ntCovu5ufjSeQtJZkSxwzO0oDL0s5aq4BstpfidOTZE='\n",
      "<18:54:43> \n",
      "<18:54:43> F0.0 AVG:   OOF(roc_auc_score=0.95455)  |  Time Elapsed: 0.04106 s\n",
      "<18:54:43> F0.1 AVG:   OOF(roc_auc_score=0.96338)  |  Time Elapsed: 0.04396 s\n",
      "<18:54:44> F0.2 AVG:   OOF(roc_auc_score=0.90079)  |  Time Elapsed: 0.04339 s\n",
      "<18:54:44> F0.3 AVG:   OOF(roc_auc_score=0.97222)  |  Time Elapsed: 0.03908 s\n",
      "<18:54:44> F0.4 AVG:   OOF(roc_auc_score=0.97619)  |  Time Elapsed: 0.03897 s\n",
      "<18:54:44> F0.5 AVG:   OOF(roc_auc_score=0.90079)  |  Time Elapsed: 0.03904 s\n",
      "<18:54:44> F0.6 AVG:   OOF(roc_auc_score=0.98611)  |  Time Elapsed: 0.03699 s\n",
      "<18:54:44> F0.7 AVG:   OOF(roc_auc_score=0.98571)  |  Time Elapsed: 0.03822 s\n",
      "<18:54:44> F0.8 AVG:   OOF(roc_auc_score=0.97619)  |  Time Elapsed: 0.03699 s\n",
      "<18:54:44> F0.9 AVG:   OOF(roc_auc_score=1.00000)  |  Time Elapsed: 0.03595 s\n",
      "<18:54:44> \n",
      "<18:54:44> FINAL:    OOF(roc_auc_score=0.96145)  |  Time Elapsed: 0.40557 s\n",
      "<18:54:44> \n",
      "<18:54:44> Saving results for Experiment: 'c3ddb52c-ac47-46bd-9b11-f6407f7ca611'\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "experiment = CVExperiment(\n",
    "    model_initializer=XGBClassifier,\n",
    "    model_init_params=dict(objective=\"reg:linear\", max_depth=3, n_estimators=100, subsample=0.5),\n",
    "    model_extra_params=dict(\n",
    "        fit=dict(\n",
    "            eval_set=[(env.train_input, env.train_target), (env.validation_input, env.validation_target)],\n",
    "            early_stopping_rounds=5,\n",
    "            eval_metric=\"mae\",\n",
    "        ),\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validated Environment with key: \"LOKr-L2lRGe8e8l0E-8SY5FEcl2u7dwBRtrl36HDjvQ=\"\n",
      "\u001b[31mSaved Result Files\u001b[0m\n",
      "\u001b[31m___________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   (fit, eval_metric) |   booster |   learning_rate |   max_depth | \n",
      "Experiments matching cross-experiment key/algorithm: 1\n",
      "Experiments fitting in the given space: 1\n",
      "Experiments matching current guidelines: 1\n",
      "    0 | c3ddb52c | 00m00s | \u001b[35m   0.96145\u001b[0m | \u001b[32m                 mae\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.1000\u001b[0m | \u001b[32m          3\u001b[0m | \n",
      "\u001b[31mHyperparameter Optimization\u001b[0m\n",
      "\u001b[31m___________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   (fit, eval_metric) |   booster |   learning_rate |   max_depth | \n",
      "    1 | 7239620d | 00m00s |    0.95298 |                  mae |      dart |          0.1864 |          17 | \n",
      "    2 | a54f1e46 | 00m00s |    0.94258 |                  mae |    gbtree |          0.4081 |          15 | \n",
      "    3 | 8b94f62d | 00m00s | \u001b[35m   0.96189\u001b[0m | \u001b[32m                 auc\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.2983\u001b[0m | \u001b[32m         13\u001b[0m | \n",
      "    4 | 3026abea | 00m00s |    0.94870 |                  mae |      dart |          0.2209 |          14 | \n",
      "    5 | ddf63034 | 00m02s |    0.95762 |                  mae |      dart |          0.0251 |          13 | \n",
      "    6 | 2d3eaea4 | 00m00s |    0.93890 |                  auc |      dart |          0.0465 |          18 | \n",
      "    7 | 445fe54a | 00m00s |    0.94170 |                  mae |    gbtree |          0.4997 |          10 | \n",
      "    8 | a861990a | 00m00s |    0.94966 |                  mae |    gbtree |          0.4054 |           5 | \n",
      "    9 | c5659543 | 00m00s |    0.94170 |                  auc |    gbtree |          0.1543 |           5 | \n",
      "   10 | 7844554b | 00m00s |    0.94450 |                  auc |      dart |          0.4997 |           2 | \n",
      "   11 | 11034374 | 00m00s |    0.93934 |                  mae |    gbtree |          0.0001 |           2 | \n",
      "   12 | 61248a8a | 00m00s |    0.93322 |                  auc |      dart |          0.4930 |          20 | \n",
      "   13 | 58e80725 | 00m00s |    0.95069 |                  auc |      dart |          0.0029 |           2 | \n",
      "   14 | 32294ec5 | 00m00s |    0.95113 |                  auc |    gbtree |          0.0046 |          20 | \n",
      "   15 | 498a0e91 | 00m01s |    0.95062 |                  mae |    gbtree |          0.0001 |          20 | \n",
      "   16 | 1011c54d | 00m00s |    0.93182 |                  auc |      dart |          0.4934 |          20 | \n",
      "   17 | 60785b1e | 00m00s |    0.95209 |                  auc |      dart |          0.0017 |           2 | \n",
      "   18 | 9778f9a7 | 00m02s |    0.95246 |                  mae |      dart |          0.0041 |          20 | \n",
      "   19 | eb96fc66 | 00m00s |    0.94406 |                  mae |      dart |          0.4964 |           3 | \n",
      "   20 | 50ae43f6 | 00m00s |    0.94170 |                  mae |    gbtree |          0.4987 |          20 | \n",
      "   21 | 96548464 | 00m01s |    0.94546 |                  mae |      dart |          0.0001 |           2 | \n",
      "   22 | 14522971 | 00m00s |    0.93698 |                  auc |    gbtree |          0.4998 |           2 | \n",
      "   23 | c864c1a4 | 00m00s |    0.94266 |                  auc |      dart |          0.0006 |          20 | \n",
      "   24 | a7d7c6e4 | 00m00s |    0.95909 |                  mae |    gbtree |          0.2157 |          20 | \n",
      "   25 | 696d0afd | 00m00s |    0.96094 |                  mae |    gbtree |          0.2213 |          20 | \n",
      "   26 | 41b89066 | 00m00s |    0.93794 |                  auc |      dart |          0.2527 |           2 | \n",
      "   27 | 60f955e3 | 00m01s |    0.95578 |                  mae |      dart |          0.2336 |          20 | \n",
      "   28 | 4ada2783 | 00m00s |    0.95017 |                  auc |    gbtree |          0.3690 |           2 | \n",
      "   29 | 7b59cd36 | 00m01s | \u001b[35m   0.96661\u001b[0m | \u001b[32m                 mae\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.1013\u001b[0m | \u001b[32m         20\u001b[0m | \n",
      "   30 | 49013e3b | 00m01s |    0.96565 |                  mae |    gbtree |          0.1037 |          20 | \n",
      "Optimization loop completed in 0:00:24.677463\n",
      "Best score was 0.9666111727709953 from Experiment \"7b59cd36-e420-471d-8a11-ec27f885ebfb\"\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import BayesianOptPro, Real, Integer, Categorical\n",
    "\n",
    "optimizer = BayesianOptPro(iterations=30, random_state=1337)\n",
    "\n",
    "optimizer.forge_experiment(\n",
    "    model_initializer=XGBClassifier,\n",
    "    model_init_params=dict(\n",
    "        objective=\"reg:linear\",\n",
    "        max_depth=Integer(2, 20),\n",
    "        learning_rate=Real(0.0001, 0.5),\n",
    "        subsample=0.5,\n",
    "        booster=Categorical([\"gbtree\", \"dart\"]),\n",
    "    ),\n",
    "    model_extra_params=dict(\n",
    "        fit=dict(\n",
    "            eval_set=[(env.train_input, env.train_target), (env.validation_input, env.validation_target)],\n",
    "            early_stopping_rounds=5,\n",
    "            eval_metric=Categorical([\"auc\", \"mae\"]),\n",
    "        ),\n",
    "    ),\n",
    ")\n",
    "\n",
    "optimizer.go()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search space/guidelines set for `optimizer`.\n",
    "\n",
    "Then, when optimization is started, it automatically learns from `experiment`'s results - without any extra work for us!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
